package datacollection;

import gui.BotGui;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.SyncFailedException;
import java.net.URL;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import bot.Util;

/**
 * Gets and formats information from web sites
 * @author Jamison Bradley
 */
public class WebParser {
	
	/**
	 * Gets the page source info from a given url.
	 * @param url is the web address of the site whose source code should be retrieved.
	 * @return the source code for the given web page.
	 * @throws IOException
	 */
	private BufferedReader read(String url) throws IOException{
		return new BufferedReader(
			new InputStreamReader(
				new URL(url).openStream()));
	}
	
	/**
	 * Reads the info from a url to a wikpedia edit page for cities and picks out the edit information.
	 * @param url is the url to the wikipedia edit page.
	 * @return A string containing the edit information for the wikipedia page.
	 * @throws IOException
	 */
	public String getWikiEditSource(String url) throws IOException{
		String ret = "";
		BufferedReader source = read(url);
		//find the start of the wiki page source by first looking for an input tag
		String line = source.readLine();
		while (line != null){
			//if the line is long enough test to see if it is an input tag
			if (line.length() > 6){
				//if it is the start of an input tag break the loop
				if (line.substring(0, 6).equals("<input")){
					break;
				}
			}
			line = source.readLine();
		}
		//if the entire file was checked without finding the input opening tag it wasn't a valid Wikpedia edit url
		if (line == null){
			return null;
		}
		//search for {{ which will be the start of source we want to pick out
		boolean exit = false;
		while (line != null  && !exit){
			if (line.length() > 2){
				//skim through the line looking for {{
				for (int i = 2; i <= line.length(); i++){
					if (line.substring(i - 2, i).equals("{{")){
						//the {{ was found so add rest of line to return string
						ret += formatLine(line.substring(i-2, line.length())) + "\n";
						line = source.readLine();
						//if line is null at this point not a valid Wikipedia edit url
						if (line == null){
							return null;
						}
						//now continue until the end of the wiki edit info is found
						while (line != null){
							if (line.length() > 11){
								//if the textarea end tag is found break out of loop since end of edit info has been found
								if (line.substring(0, 11).equals("</textarea>")){
									break;
								}
							}
							//end hasn't been found yet so add the line the return string
							ret += formatLine(line) + "\n";
							line = source.readLine();
						}
						//break out of outer loop since the info needed has been parsed out
						exit = true;
						break;
					}
				}
			}
			line = source.readLine();
		}
		//one of the loops never ended meaning its not a valid Wikipedia edit url
		if (line == null){
			return null;
		}
		else{
			return ret;
		}
	}
	
	/**
	 * Parses the source code of a US Census page to grab all the needed data to update the wikipedia page.
	 * @param f is the File containing the source code of the page.
	 * @return Returns the info needed in one line.
	 */
	public String parseCensusPageData(File f){
		String ret = "";
		try{
			int infoFound = 0;
			BufferedReader reader = new BufferedReader(new FileReader(f));
			String line = reader.readLine();
			//scan the whole document and pick out relevant info
			while (line != null){
				//pick out the name of the city
				if (line.length() > 31 && line.substring(16, 32).equals("<div id=\"table1\"")){
					//can skip over all the stuff before options start
					int i = 1075;
					while (i < line.length() - 19){
						if (line.substring(i, i + 19).equals("selected=\"selected\"")){
							//look for the comma between the name and state
							int start = i + 20;
							int end = 0;
							for (int j = start; j < line.length(); j++){
								if (line.charAt(j) == ','){
									end = j;
									break;
								}
							}
							//parse out the town name
							String toAdd = line.substring(start, end);
							for (int j = toAdd.length() - 1; j >= 0; j--){
								if (toAdd.charAt(j) == ' '){
									ret += toAdd.substring(0, j) + " ;&: " + toAdd.substring(j + 1, toAdd.length());
									break;
								}
							}
//							ret += line.substring(start, end);
						}
						i++;
					}
				}
				//get the info from the table
				else if (line.length() > 9 && line.substring(0, 9).equals("<tr class")){
					//looking for total population
					String test = null;
					if (infoFound == 0){
						test = getLineInfo(line, "Total population", 2);
					}
					//get the other age data
					else if (infoFound == 1){
						test = getAgeData(line, reader);
					}
					//get the male percentage of the population
					else if (infoFound == 2){
						test = getLineInfo(line, "Male population", 4);
					}
					//get the female percentage of the population
					else if (infoFound == 3){
						test = getLineInfo(line, "Female population", 4);
					}
					//get the percentage of the population that is one race White
					else if (infoFound == 4){
						test = getLineInfo(line, "White", 4);
					}
					//get the percentage of the population that is one race African American
					else if (infoFound == 5){
						test = getLineInfo(line, "Black or African American", 4);
					}
					//get the percentage of the population that is one race American Indian
					else if (infoFound == 6){
						test = getLineInfo(line, "American Indian and Alaska Native", 4);
					}
					//get the percentage of the population that is one race Asian
					else if (infoFound == 7){
						test = getLineInfo(line, "Asian", 4);
					}
					//get the percentage of the population that is one race Pacific Islander
					else if (infoFound == 8){
						test = getLineInfo(line, "Native Hawaiian and Other Pacific Islander", 4);
					}
					//get the percentage of the population that is an other race
					else if (infoFound == 9){
						test = getLineInfo(line, "Some Other Race", 4);
					}
					//get the percentage of the population that is of two or more races
					else if (infoFound == 10){
						test = getLineInfo(line, "Two or More Races", 4);
					}
					//get the percentage of the population that is hispanic of any race
					else if (infoFound == 11){
						test = getLineInfo(line, "Hispanic or Latino (of any race)", 4);
					}
					//get the number of households
					else if (infoFound == 12){
						test = getLineInfo(line, "Total households", 2);
					}
					//get the number of families
					else if (infoFound == 13){
						test = getLineInfo(line, "Family households (families) [7]", 2);
					}
					//get the percentage of married couples
					else if (infoFound == 14){
						test = getLineInfo(line, "Husband-wife family", 4);
					}
					//get percentage of male householders no wife
					else if (infoFound == 15){
						test = getLineInfo(line, "Male householder, no wife present", 4);
					}
					//get percentage of female householders with no husband
					else if (infoFound == 16){
						test = getLineInfo(line, "Female householder, no husband present", 4);
					}
					//get the percentage of non family households
					else if (infoFound == 17){
						test = getLineInfo(line, "Nonfamily households [7]", 4);
					}
					//get the percentage of people living alone
					else if (infoFound == 18){
						test = getLineInfo(line, "Householder living alone", 4);
					}
					//get the percentage of people living alone over 65 years old
					else if (infoFound == 19){
						test = getLivingAloneOver65(line, reader);
					}
					//get the percentage of households with children under 18
					else if (infoFound == 20){
						test = getLineInfo(line, "Households with individuals under 18 years", 4);
					}
					//get the average household size
					else if (infoFound == 21){
						test = getLineInfo(line, "Average household size", 2);
					}
					//get the average family size
					else if (infoFound == 22){
						test = getLineInfo(line, "Average family size [7]", 2);
					}
					//get the total number of housing units
					else if (infoFound == 23){
						test = getLineInfo(line, "Total housing units", 2);
					}
					//exit at this point done finding data
					else{
						break;
					}
						
					//if test is null the value wasn't in that line
					if (test != null){
						ret += " ;&: " + test;
						infoFound++;
					}
				}
				line = reader.readLine();
			}
		} 
		catch (FileNotFoundException e){
			e.printStackTrace();
		} 
		catch (IOException e){
			e.printStackTrace();
		}
		return ret;
	}
	
	/**
	 * Parses out the info in the page source code of the Us Census page data.
	 * @param source is the line that is being parsed.
	 * @param keyword is the keyword that is being looked for to determine if the line is valid or not.
	 * @return The number for that line if the keyword if found or it will return null if the keyword wasn't in that line.
	 */
	private String getLineInfo(String source, String keyword, int offset){
		String ret = null;
		//check the input to make sure it is big enough to be scanned
		if (source == null || source.length() < keyword.length()){
			throw new IllegalArgumentException("Size of source String is too small for keyword");
		}
		//search for the keyword in the source
		for (int i = 0; i < source.length() - keyword.length(); i++){
			//keyword found so now parse out the string the second > and third < from the keyword
			if (source.substring(i, i + keyword.length()).equals(keyword)){
				int count = 0;
				int start = 0;
				int end = 0;
				for (int j = i + keyword.length(); j < source.length(); j++){
					//parse and break
					if (count == offset){
						if (source.charAt(j) == '<'){
							end = j;
							ret = source.substring(start, end);
							return ret;
						}
					}
					else if (source.charAt(j) == '>'){
						count++;
						start = j + 1;
					}
				}
			}
		}
		return ret;
	}
	
	/**
	 * Gets the percentages of the population from the page source.
	 * @param current is the line that the scan is starting on.
	 * @param reader is the source to get the next lines in the table.
	 * @return under 18  18-24  25-44  45-64  over 65  in that order. 
	 */
	private String getAgeData(String current, BufferedReader reader){
		String ret = "";
		String line = current;
		int infoCount = 0;
		String test = null;
		double[] values = new double[16];
		//get needed info for the age data
		while (line.length() > 10){
			//get data on people under 5 years old
			if (infoCount == 0){
				test = getLineInfo(line, "Under 5 years", 4);
			}
			//get data on people between 5 and 64
			else if (infoCount > 0 && infoCount <= 12){
				test = getLineInfo(line, (infoCount * 5) + " to " + ((infoCount + 1) * 5 - 1), 4);
			}
			//get median age
			else if (infoCount == 13){
				test = getLineInfo(line, "Median age (years)", 2);
			}
			//get data for number of people over 18 years old
			else if (infoCount == 14){
				test = getLineInfo(line, "18 years and over", 4);
			}
			//get data for number of people over 65 years old
			else if (infoCount == 15){
				test = getLineInfo(line, "65 years and over", 4);
			}
			//quit search
			else{
				break;
			}
			
			//add the value retrieved from the table to the values array
			if (test != null){
				values[infoCount] = Double.parseDouble(test);
				infoCount++;
			}
			//go to next line
			try {
				line = reader.readLine();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		DecimalFormat df = new DecimalFormat("#.#");
		//add the under 18 info to ret
		double under18 = 100 - values[14];
		if (under18 == 0){
			ret += "0.0" + " ;&: ";
		}
		else{
			ret += df.format(under18) + " ;&: ";
		}
		//add the info for 18 to 24 to ret
		double under20 = values[0] + values[1] + values[2] + values[3];
		double range18to24 = under20 - under18 + values[4];
		if (range18to24 == 0 || range18to24 < 0){
			ret += "0.0" + " ;&: ";
		}
		else{
			ret += df.format(range18to24) + " ;&: ";
		}
		//add the info for 25 to 44 to ret
		double range25to44 = values[5] + values[6] + values[7] + values[8];
		if (range25to44 == 0){
			ret += "0.0" + " ;&: ";
		}
		else{
			ret += df.format(range25to44) + " ;&: ";
		}
		//add the info for 45 to 64 to ret
		double range45to64 = values[9] + values[10] + values[11] + values[12];
		if (range45to64 == 0){
			ret += "0.0" + " ;&: ";
		}
		else{
			ret += df.format(range45to64) + " ;&: ";
		}
		//add 65 and over data
		double over65 = values[15];
		if (over65 == 0){
			ret += "0.0" + " ;&: ";
		}
		else{
			ret += df.format(over65) + " ;&: ";
		}
		//add the median age info
		double medianAge = values[13];
		if (medianAge == 0){
			ret += "0.0";
		}
		else{
			ret += df.format(medianAge);
		}
		return ret;
	}
	
	/**
	 * Calculates the percentage of the people in the area that are over 65 years old and living alone.
	 * @param current is the line that the scan is starting on.
	 * @param reader is the source to get the next lines in the table.
	 * @return The percentage of the population over 65 years old and living alone.
	 */
	private String getLivingAloneOver65(String current, BufferedReader reader){
		String ret = "";
		String line = current;
		String test = null;
		int infoCount = 0;
		double[] values = new double[2];
		while (line.length() > 10){
			//get the percentage of males over 65 living alone
			if (infoCount == 0 || infoCount == 1){
				test = getLineInfo(line, "65 years and over", 4);
			}
			//quit
			else{
				break;
			}
			
			
			//add the value retrieved from the table to the values array
			if (test != null){
				values[infoCount] = Double.parseDouble(test);
				infoCount++;
			}
			//go to next line
			try {
				line = reader.readLine();
			} catch (IOException e) {
				e.printStackTrace();
			}	
		}
		//calculate the percentage and return
		double singleOver65 = values[0] + values[1];
		DecimalFormat df = new DecimalFormat("#.#");
		if (singleOver65 == 0){
			ret += "0.0";
		}
		else{
			ret += df.format(singleOver65);
		}
		return ret;
	}
	
	
	/**
	 * Finds "&lt;" and replaces it with < and "&amp;" with a &
	 * @param line is the String that needs the "&lt;"'s replaced
	 * @return String that has the "&lt;"'s replaced with <
	 */
	private String formatLine(String line){
		//look for "&lt;" which needs to be converted to a <
		if (line != null && line.length() > 3){
			for (int i = 4; i <= line.length(); i++){
				//if the string is found replace it with a <
				if (line.substring(i - 4, i).equals("&lt;")){
					line = line.substring(0, i - 4) + "<" + line.substring(i, line.length());
					//change i so substrings don't get skipped
					if (i >= 7){
						i -= 3;
					}
					else{
						i = 4;
					}
				}
				else if (i > 4 && line.substring(i - 5, i).equals("&amp;")){
					line = line.substring(0, i - 5) + "&" + line.substring(i, line.length());
					//change i so substrings don't get skipped
					if (i >= 8){
						i -= 4;
					}
					else{
						i = 4;
					}
				}
			}
		}
		return line;
	}
	
	/**
	 * Retrieves the urls for the cities in the given county url.
	 * @param url Wikipedia page of the county whose cities you want to retrieve.
	 * @return A list of cities located in the given county url.
	 */
	public List<String> getCitiesInCounty(String url){
		List<String> ret = new ArrayList<String>();
		try {
			BufferedReader input = read(url);
			String line = input.readLine();
			boolean foundCity = false;
			boolean foundTown = false;
			boolean foundVillage = false;
			while (line != null){
				if (line.split("title=\"City\"").length > 1 || line.split("title=\"City \\(" + BotGui.state + "\\)").length > 1 ||
						line.split("title=\"City \\(United States\\)").length > 1 || line.split("title=\"List of cities in " + BotGui.state).length > 1 ||
						line.split("title=\"List of incorporated cities in " + BotGui.state).length > 1 ||
						line.split("title=\"List of cities and unincorporated communities in " + BotGui.state + "\">Cities").length > 1){
					foundCity = true;
				}
				if (foundCity && line.length() > 4 && "<li>".equals(line.substring(0, 4))){
					ret.add(line.split("\"")[1]);
				}
				//look for </ul> to end the loop so extra junk doesn't get added to the list
				else if (foundCity && line.length() >= 5 && "</ul>".equals(line.substring(0, 5))){
					foundCity = false;
				}
				//check for town section
				if (line.split("title=\"Town\"").length > 1 ||line.split("title=\"Town \\(" + BotGui.state + "\\)").length > 1 ||
						line.split("title=\"Town \\(United States\\)").length > 1 || line.split("title=\"Incorporated town\"").length > 1 ||
						line.split("title=\"New England town\"").length > 1 || line.split("title=\"List of towns in " + BotGui.state).length > 1){
					foundTown = true;
				}
				if (foundTown && line.length() > 4 && "<li>".equals(line.substring(0, 4))){
					try{
						ret.add(line.split("\"")[1]);
					}
					catch (ArrayIndexOutOfBoundsException e){
						System.out.println("problem with   " + line);
					}
				}
				else if (foundTown && line.length() >= 5 && "</ul>".equals(line.substring(0, 5))){
					foundTown = false;
				}
				//check for village section
				if (line.split("title=\"Village\"").length > 1 ||line.split("title=\"Village \\(" + BotGui.state + "\\)").length > 1 ||
						line.split("title=\"Village \\(United States\\)").length > 1){
					foundVillage = true;
				}
				if (foundVillage && line.length() > 4 && "<li>".equals(line.substring(0, 4))){
						ret.add(line.split("\"")[1]);
				}
				else if (foundVillage && line.length() >= 5 && "</ul>".equals(line.substring(0, 5))){
					foundVillage = false;
				}
				line = input.readLine();
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return ret;
	}
	
	/**
	 * Retrieves the urls for the cities in the given county url.
	 * @param url Wikipedia page of the county whose cities you want to retrieve.
	 * @return A list of cities located in the given county url.
	 */
	public List<String> getCitiesInState(String url){
		List<String> ret = new ArrayList<String>();
		Set<String> cities = new TreeSet<String>();
		List<String> counties = new ArrayList<String>();
		try{
			BufferedReader input = read(url);
			String line = input.readLine();
			boolean found = false;
			while (line != null){
				if (line.length() > 3 && "<th".equals(line.substring(0, 3)) && (line.split("title=\"List of counties in").length > 1 ||
						line.split("title=\"List of " + BotGui.state + " counties").length > 1)){
					found = true;
				}
				if (found && line.length() > 4 && "<li>".equals(line.substring(0, 4))){
					counties.add(line.split("\"")[1]);
				}
				//look for </ul> to end the loop so extra junk doesn't get added to the list
				else if (found && line.length() >= 5 && "</ul>".equals(line.substring(0, 5))){
					break;
				}
				line = input.readLine();
			}
			//cycle through all counties and get towns
			for (String county: counties){
				List<String> townsInCounty = getCitiesInCounty("http://en.wikipedia.org" + county);
				for (String town: townsInCounty){
					try{
						if (cities.add(Util.convertToEditUrl(town))){
							ret.add(Util.convertToEditUrl(town) + "\n");
						}
					}
					catch (ArrayIndexOutOfBoundsException e){
						System.out.println("Problem adding " + town);
					}
				}
			}
		} 
		catch (IOException e){
			e.printStackTrace();
		}
		return ret;
	}
}
